"""
一、数据来源分析
    1、需求分析
        - 今日头条推荐页面的，详情链接，
        - 进入新闻详情后，采集新闻标题，发布时间，发布的媒体，新闻内容
    2、接口分析
        - 分析是静态数据还是动态数据

二、爬虫代码实现
    1、发送请求
    2、获取数据
    3、解析数据
    4、保存数据
"""
import requests
from selenium import webdriver
import time
import parsel
from selenium.webdriver.common.by import By


class ToutiaoCrawler:

    def __init__(self):
        self.url = 'https://www.toutiao.com/api/pc/list/feed'
        self.params = {
            "channel_id": "0",
            "min_behot_time": "0",
            "offset": "0",
            "refresh_count": "1",
            "category": "pc_profile_recommend",
            "aid": "24",
            "app_name": "toutiao_web",
            # 这个签名会有时效，需要逆向
            "_signature": "_02B4Z6wo009011BN1rQAAIDCM0cW3H4f5fNQadIAALFd2nEZbs-bFIIcFh6p5cu3.Rtl.fuMgiCnC0MxuCkXg73NRFGhU96IhQ5L-XSyvLej0JYYpJoUWOrerNdqeN6DZ8lfCTpOl23Vs5iP62"
        }
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
        }
        self.recommend_data = {}
        self.chrome = webdriver.Chrome()

    def get_recommend(self):
        try:
            resp = requests.get(url=self.url, params=self.params, headers=self.headers)
            self.recommend_data = resp.json()
        except Exception as e:
            print(e)

    def get_journalism(self):
        journalism = self.recommend_data['data']
        for news in journalism:
            title = news['title']
            news_url = news['url']
            try:
                if news_url.find('https://toutiao.com/group') != -1:
                    print(title, news_url)
                    self.chrome.get(news_url)
                    time.sleep(5)
                    p_content = self.chrome.find_elements(By.XPATH, '//div[@class="article-content"]//p[@data-track]')
                    print('\n'.join([x.text for x in p_content]))
                    time.sleep(3)
            except Exception as e:
                print(e)

    def run(self):
        self.get_recommend()
        self.get_journalism()
        self.chrome.close()


if __name__ == '__main__':
    spider = ToutiaoCrawler()
    spider.run()
